const puppeteer = require('puppeteer'); //  引入依赖  
//const fs = require('fs');
//agent
const superagent = require("superagent");
//require("superagent-proxy")(superagent);
const charset = require("superagent-charset");
const cheerio = require('cheerio');
const userAgent = require('../../tools/userAgent');
const delay = require('../../tools/delay');
const retry = require('../../tools/retry')
//mongo
const format = require('../../tools/format')();
const config = require('../mongodb.config');
const db = require('../../tools/mongodb');
//const nodePath = require('path');
// const userAgent = require(nodePath.resolve('./tools/userAgent'));
// const delay = require(nodePath.resolve('./tools/delay'));
// const retry = require(nodePath.resolve('./tools/retry'));
// //mongo
// const format = require(nodePath.resolve('./tools/format'))();
// const config = require(nodePath.resolve('./jd/mongodb.config'));
// const db = require(nodePath.resolve('./tools/mongodb'));


const cutText = (start, end, text) => {
    let tmp = text.substring((text.indexOf(start) + start.length));
    return tmp.substring(0, tmp.indexOf(end));
}



let shopSearch = (async () => {
    //define
    const startTime = new Date().format("yyyy-MM-dd HH:mm:ss");
    const collection_shops = config.jd.shops;
    const collection_items = config.jd.items;
    const collection_log = config.jd.shopsLog;
    const collection_error = config.jd.shopsError;
    let logId = Number(new Date(startTime));
    let daysOfYear = parseInt(Math.ceil(logId-Number(new Date(`${new Date().getFullYear()}-01-01`))));
    const timeout = 1800 * 1000;
    const shortTimeout = 900 * 1000;
    //main
    const browser = await (puppeteer.launch({ headless: false }));//headless:false 时会打开浏览器界面
    const page = await browser.newPage();
    await page.evaluateOnNewDocument(() => {
        const newProto = navigator.__proto__;
        delete newProto.webdriver;
        navigator.__proto__ = newProto;
    });
    //设置可视区域大小
    await page.setViewport({ width: 1600, height: 1080 });
    let shops = [];
    let current = {};//全局（函数内）记录当前处理的店铺/分类
    try {
        await db.find(collection_shops, { disabled: { $ne: "1" } }).then(r => {
            shops = r;
        }).catch(error => { throw error });
        // //设置每个店铺独立爬取间隔————2020年3月19日 取消间隔设置————2020年5月25日
        // if(!item.everyFewDays || item.everyFewDays == 0 || Number.isNaN(item.everyFewDays)||Number.isFinite(item.everyFewDays)){
        //     item.everyFewDays = 1;
        // }
        if (shops.length > 0 && shops[0].shopId) {
            await db.deleteMany(collection_items, {}).catch(error => { throw error });//清空商品列表
        }
        
        let _idArray =  shops.map((item)=>{
            return item._id;
        });
        shops = null;//存储_id 减少内存占用————2020年3月20日
        await db.insertOne(collection_log, {
            'operation': "crawl",
            'logId': logId,
            'start': startTime,
            'daysOfYear':daysOfYear,
            'context':`本次任务需处理${_idArray.length}个店铺`,
        }).catch(error => { throw error });
        for (let _id of _idArray) {
            let shop = {};
            await db.find(collection_shops,{'_id':_id}).then(r=>{
                shop = r[0];
            }).catch(error=>{throw error;});
            let homePage = `https://mall.jd.com/index-${shop.shopId}.html`;
            if (!shop.keywords || shop.keywords.length == 0) {
                continue;
            }
            current ={
                '_id':_id,
                'shop':shop.shopId,
                'homepage':homePage
            };
            let shopInfo = { 'url': homePage };//先获取商店首页数据，容后更新
            for (let keyword of shop.keywords) {
                current.keyword = keyword;
                await page.goto("about:blank", { 'timeout': timeout });
                await page.goto(homePage, { 'timeout': timeout });
                await page.waitForSelector('.button01', { 'timeout': timeout });
                let search = await page.$('.button01');
                // let input = await page.$('#key');
                // await input.type('');
                shopInfo.title = await page.evaluate((keyword) => {
                    //搜索栏填充
                    let key = (document.querySelector("#key") || document.querySelector("#key01"));
                    key.value = keyword;
                    return document.querySelector("title").innerText.trimRight().trimLeft();
                }, keyword).catch(error => { throw error; });
                await search.click();
                //使用retry控制易超时部分 2020年3月20日
                let _main2ListPage = (async() =>{
                     await page.waitForNavigation({ 'timeout': shortTimeout });
                     await page.waitForSelector('a[orderby="1"]',{'timeout':shortTimeout});
                });
                let _reload = (async()=>{
                    await page.reload();
                });
                await retry(_main2ListPage,3,3000,_reload).catch(error=>{throw error;});
                // await page.waitForNavigation({ 'timeout': shortTimeout });
                // await page.waitForSelector('a[orderby="1"]',{'timeout':shortTimeout});
                //20200528 //shopId 644457 sort无法点击 因此跳过
                // let sort = await page.$('a[orderby="1"]');
                // await sort.click();
                // await page.waitForNavigation({ 'timeout': timeout });

                // let _main2Sorted = (async()=>{
                //     await page.waitForSelector('.jSearchListArea .jItem',{'timeout':shortTimeout});
                // });
                // await retry(_main2Sorted,3,30000,_reload).catch(error=>{throw error;});
                //await page.screenshot({ path: "list1.png" });
                // await element.screenshot({
                //     path: './pic2.png'
                // });

                let sUrlExist = [];
                let pageUrls = await page.evaluate(() => {
                    let items = [...document.querySelectorAll('.mc .jPage a')];
                    let urls = items.map((item) => { return item.href.split('?')[0]; });//去掉get参数，避免net::ERR_ABORTED 
                    urls = Array.from(new Set(urls));//去重
                    if (urls.includes("")) {
                        urls.splice(urls.indexOf(""), 1, document.URL);
                    }
                    return urls;
                });
                //pageUrls.sort((a, b) => { return a.length - b.length; });//确保''排在前面
                //console.log(pageUrls);
                for (let pUrl of pageUrls) {
                    current.pageUrl = pUrl;
                    await delay(1000);
                    //使用retry控制易超时部分 2020年3月20日
                    let _main2SinglePage = (async()=>{
                        await page.goto("about:blank", { 'timeout': shortTimeout });//终止原页面内JS的执行
                        await page.goto(pUrl, { 'timeout': shortTimeout });
                        await page.waitForSelector('.jSearchListArea .jItem',{'timeout':shortTimeout});
                    });
                    await retry(_main2SinglePage,3,30000).catch(error=>{throw error});
                    //获取items
                    let items = await page.evaluate((shop) => {
                        let items = [...document.querySelectorAll('.mc .j-module .jItem')];
                        let _array = [];
                        for (let item of items) {
                            let scroll = [...item.querySelectorAll('.jScrollWrap li')];
                            let hostSkuId = "";
                            for (let i = 0; i < scroll.length; i++) {
                                let obj = {};
                                obj.skuId = scroll[i].getAttribute('data-href').split('jd.com/')[1].split('.')[0];
                                obj.url = scroll[i].getAttribute('data-href');
                                if (i == 0) {
                                    hostSkuId = obj.skuId;
                                    obj.title = item.querySelector('.jGoodsInfo .jDesc a').innerText;
                                    obj.price = item.querySelector('.jGoodsInfo .jdPrice .jdNum').innerText;
                                    obj.comments = item.querySelector('.jGoodsInfo .jExtra .jCommentNum').innerText;
                                }
                                else {
                                    obj.title = scroll[i].querySelector('a').title;
                                    obj.price = "";
                                    obj.comments = "";
                                }
                                obj.hostSkuId = hostSkuId;
                                obj.shopId = shop.shopId;
                                obj.createTime = new Date().format("yyyy-MM-dd HH:mm:ss");
                                _array.push(obj);
                            }
                        }
                        return _array;
                    }, shop);
                    //添加keyword字段————2020年3月10日
                    items = items.map((item)=>{
                        item.keyword =  keyword;
                        return item;
                    });
                    if(items.length>0)//debug ————2020年3月20日
                    {
                        await db.insertMany(collection_items, items).catch(error => { throw error });
                    }
                }
            }
            await db.updateMany(collection_shops, { shopId: shop.shopId }, { ...shopInfo, ...{ changeTime: new Date().format("yyyy-MM-dd HH:mm:ss") } }).catch(error => { throw error });
            await delay(25000 + Math.random() * 5000);
        }
        await db.updateMany(collection_log, { logId: logId }, {
            'end': new Date().format("yyyy-MM-dd HH:mm:ss")
        }).catch(error => { throw error });
        console.log(`店铺商品列表已于[${new Date().format('yyyy-MM-dd HH:mm:ss')}]获取完毕`);
    }
    catch (error) {
        console.error(error.stack);
        await db.insertOne(collection_error, {
            'logId': logId,
            'time': new Date().format('yyyy-MM-dd HH:mm:ss'),
            'error': error,
            'stack': error.stack,
            'current':current
        });
        // fs.writeFileSync(`./err.json`, JSON.stringify({'error':err}));
    }
    await page.close();
    await browser.close();

});

module.exports = shopSearch;

